# 取句子主干
import jieba.posseg
import re

jieba.enable_paddle()  # 开启paddle模式

keep = ["n", "f", "s", "nr", "ns", "nt", "nw", "nz",
        "v", "vd", "vn", "t", "m", "q", "TIME", "ORG", "LOC", "PER", "w"]
set_not = ["不", "非", "有", "无", "莫"]


def keep_main_content(sentence):
    out = ""
    words = jieba.posseg.cut(sentence, use_paddle=True)
    for item in words:
        if item.flag in keep or item.word in set_not:
            out += item.word
    return out


def handle(in1, out1):
    fp = open(in1, 'r', encoding='utf-8')
    # 'a'是在原先的文本的基础上进行直接增加
    # 'w'会覆盖之前的文本
    fout = open(out1, 'a', encoding='utf-8')

    i = 0
    for line in fp.readlines():
        i += 1
        set_l = line.strip().split("\t")
        if bool(re.search("[-\d()*+/=\.]{15,}", set_l[0])):
            fout.write('%s' % line)
            print("这行不加噪处理：", i)
            continue
        result = keep_main_content(set_l[1])
        fout.write('%s\n' % (result + "\t" + set_l[1]))
        # if i % 100 == 0:
        #     print("执行到了：%s 行" % i)


if __name__ == '__main__':
    infile = 'g:/out3.tsv'
    outfile = 'g:/keep_content.tsv'
    handle(infile, outfile)
